#!/usr/bin/env python
#-*- coding:utf-8 -*-

import re
from unicodedata import normalize as normalize_word

from nltk.corpus import stopwords
from nltk import PorterStemmer
porter = PorterStemmer()

def build_dataset(dataset, partition=0.7):
    documents = [[list(dataset.words(fid)), cat] \
        for cat in dataset.categories()
        for fid in dataset.fileids(cat)]
    
    return documents

def normalize(word, codif='utf-8'):
    """ Removes both special and numeric characters
    
        Params:
            word: the input string
            codif: the codec for decoding the string
        Returns: 
            a lowered case and normalized string
    """
    
    # Removes all non-ascii characters (aka a shortcut to remove accentuation)
    word = normalize_word('NFKD', word.decode(codif)).encode('ASCII','ignore')
    
    # Removes all non-alpha characters
    pattern = re.compile(r'[\W_]+|[\d]+')
    word = pattern.sub('', word)
    
    return word.lower()
    
if __name__ == '__main__':
    from nltk.corpus import movie_reviews
    from nltk import PorterStemmer
    import pickle
    
    porter = PorterStemmer()
    stopwords = stopwords.words('english')
    
    docs = build_dataset(dataset=movie_reviews)
    
    # docs => [[['word1','word2','word3'], 'neg'], [['word1','word2','word3'], 'neg']]
    cleaned_docs = []
    for doc in docs:
        cleaned_words = []
        for word in doc[0]:
            word = normalize(word)
            word = porter.stem(word)
            if word and word not in stopwords:
                cleaned_words.append(word)
        cleaned_docs.append([cleaned_words, doc[-1]])
        
    print cleaned_docs[-1]
    
    with open('dataset/pos', 'w') as positive:
        positives = [doc for doc in cleaned_docs if doc[-1] == 'pos']
        print "len(positives)", len(positives)
        pickle.dump(positives, positive)
        
    with open('dataset/neg', 'w') as negative:
        negatives = [doc for doc in cleaned_docs if doc[-1] == 'neg']
        print "len(negatives)", len(negatives)
        pickle.dump(negatives, negative)
        
        
